{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于文档的问答"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA # 检索QA链，在文档上进行检索\n",
    "from langchain.document_loaders import CSVLoader #文档加载器，采用csv格式存储\n",
    "from langchain.vectorstores import DocArrayInMemorySearch # 向量存储\n",
    "from IPython.display import display, Markdown\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv()\n",
    "dashcope_api_key = os.environ.get(\"DASHSCOPE_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Women's Campside Oxfords</td>\n",
       "      <td>This ultracomfortable lace-to-toe Oxford boast...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Recycled Waterhog Dog Mat, Chevron Weave</td>\n",
       "      <td>Protect your floors from spills and splashing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Infant and Toddler Girls' Coastal Chill Swimsu...</td>\n",
       "      <td>She'll love the bright colors, ruffles and exc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Refresh Swimwear, V-Neck Tankini Contrasts</td>\n",
       "      <td>Whether you're going for a swim or heading out...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EcoFlex 3L Storm Pants</td>\n",
       "      <td>Our new TEK O2 technology makes our four-seaso...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                name  \\\n",
       "0                           Women's Campside Oxfords   \n",
       "1           Recycled Waterhog Dog Mat, Chevron Weave   \n",
       "2  Infant and Toddler Girls' Coastal Chill Swimsu...   \n",
       "3         Refresh Swimwear, V-Neck Tankini Contrasts   \n",
       "4                             EcoFlex 3L Storm Pants   \n",
       "\n",
       "                                         description  \n",
       "0  This ultracomfortable lace-to-toe Oxford boast...  \n",
       "1  Protect your floors from spills and splashing ...  \n",
       "2  She'll love the bright colors, ruffles and exc...  \n",
       "3  Whether you're going for a swim or heading out...  \n",
       "4  Our new TEK O2 technology makes our four-seaso...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = \"../data/OutdoorClothingCatalog_1000.csv\"\n",
    "# 使用langchain文档加载器对数据进行导入\n",
    "loader = CSVLoader(file_path=file)\n",
    "# 使用pandas导入数据\n",
    "data = pd.read_csv(file, usecols=[1,2])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 基本文档加载器创建向量存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dashscope\n",
    "from langchain_community.embeddings import DashScopeEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入向量存储索引创建器\n",
    "from langchain.indexes import VectorstoreIndexCreator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 千问 embedding 模型\n",
    "dashscope.base_http_api_url = \"https://dashscope.aliyuncs.com/api/v1\"\n",
    "embedding = DashScopeEmbeddings(model=\"text-embedding-v3\",\n",
    "                                dashscope_api_key=dashcope_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/debian/code/AIAgent_learning/Langchain-learning/.venv/lib/python3.12/site-packages/pydantic/_migration.py:283: UserWarning: `pydantic.error_wrappers:ValidationError` has been moved to `pydantic:ValidationError`.\n",
      "  warnings.warn(f'`{import_path}` has been moved to `{new_location}`.')\n"
     ]
    }
   ],
   "source": [
    "# 创建指定向量存储类，创建完成后，从加载器中调用，通过文档加载器列表加载\n",
    "index = VectorstoreIndexCreator(vectorstore_cls=DocArrayInMemorySearch, embedding=embedding).from_loaders([loader])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查询创建的向量存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"请用markdown表格的方式列出具有防晒功能的衬衫，对每件衬衫描述进行总结\"\n",
    "# 使用索引查询和创建一个响应，并传入这个查询\n",
    "llm = ChatOpenAI(base_url=\"https://dashscope.aliyuncs.com/compatible-mode/v1\",model=\"qwen-flash\",api_key=dashcope_api_key)\n",
    "response = index.query(query, llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "| 名称 | 防晒等级 | 材质 | 主要特点总结 |\n",
       "|------|----------|------|----------------|\n",
       "| Sunrise Tee | UPF 50+ | 71%尼龙 + 29%聚酯纤维（外层），100%聚酯纤维（内衬） | 轻盈透气，快速排汗且速干，专为炎热天气设计；适合钓鱼或旅行，具有优异的紫外线防护和抗皱性能。 |\n",
       "| Performance Plus Woven Shirt | UPF 40+（宣称SPF 50+） | 100%尼龙 | 外观如棉质，但具备高性能速干、耐磨、抗紫外线功能；14分钟内即可干燥，适合户外与城市穿着。 |\n",
       "| Perform-A-Tex Woven Shirt | UPF 40+（宣称SPF 50+） | 100%尼龙 | 与Performance Plus同款，具备快速干燥、吸湿排汗、抗磨损特性；防晒效果持久不褪色。 |\n",
       "| Girls' Ocean Breeze Long-Sleeve Stripe Shirt | UPF 50+ | 尼龙 Lycra®-氨纶混纺 | 全覆盖防晒长袖衫，适合泳池或海滩使用；高防晒等级，经皮肤癌基金会推荐，耐海水、不易褪色，保型性好。 |\n",
       "\n",
       "> ✅ 所有产品均提供**UPF 50+**或等效**SPF 50+**防晒保护，可有效阻挡98%的有害紫外线。  \n",
       "> 🌞 特别推荐用于户外活动、海滩度假或长时间日晒环境。"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(Markdown(response))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 结合表征模型和向量存储"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建一个文档加载器，通过csv格式加载\n",
    "file = \"../data/OutdoorClothingCatalog_1000.csv\"\n",
    "loader = CSVLoader(file)\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'source': '../data/OutdoorClothingCatalog_1000.csv', 'row': 0}, page_content=\": 0\\nname: Women's Campside Oxfords\\ndescription: This ultracomfortable lace-to-toe Oxford boasts a super-soft canvas, thick cushioning, and quality construction for a broken-in feel from the first time you put them on. \\n\\nSize & Fit: Order regular shoe size. For half sizes not offered, order up to next whole size. \\n\\nSpecs: Approx. weight: 1 lb.1 oz. per pair. \\n\\nConstruction: Soft canvas material for a broken-in feel and look. Comfortable EVA innersole with Cleansport NXT® antimicrobial odor control. Vintage hunt, fish and camping motif on innersole. Moderate arch contour of innersole. EVA foam midsole for cushioning and support. Chain-tread-inspired molded rubber outsole with modified chain-tread pattern. Imported. \\n\\nQuestions? Please contact us for any inquiries.\")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看单个文档，每个文档对应于csv中的一行\n",
    "docs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 文本向量表征模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[32m向量表征的长度：\u001b[0m \n",
      " 1024\n",
      "\n",
      "\u001b[32m向量表征的前五个元素：\u001b[0m \n",
      " [-0.07498032599687576, 0.04016530141234398, -0.056521862745285034, -0.04784677177667618, -0.039209894835948944]\n"
     ]
    }
   ],
   "source": [
    "# 使用千问提供的embedding类\n",
    "embeddings = DashScopeEmbeddings(model=\"text-embedding-v3\", dashscope_api_key=dashcope_api_key)\n",
    "# 因为文档比较短了，所以这里不需要进行任何分块，可以直接进行向量表征\n",
    "embed = embeddings.embed_query(\"你好啊，我的名字叫小可爱\")\n",
    "# 查看得到向量表征的长度\n",
    "print(\"\\n\\033[32m向量表征的长度：\\033[0m \\n\",len(embed))\n",
    "print(\"\\n\\033[32m向量表征的前五个元素：\\033[0m \\n\",embed[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 基于向量表征创建并查询向量存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[32m返回文档的个数：\u001b[0m \n",
      " 4\n",
      "\n",
      "\u001b[32m返回的第一个文档：\u001b[0m \n",
      " page_content=': 618\n",
      "name: Men's Tropical Plaid Short-Sleeve Shirt\n",
      "description: Our lightest hot-weather shirt is rated UPF 50+ for superior protection from the sun's UV rays. With a traditional fit that is relaxed through the chest, sleeve, and waist, this fabric is made of 100% polyester and is wrinkle-resistant. With front and back cape venting that lets in cool breezes and two front bellows pockets, this shirt is imported and provides the highest rated sun protection possible. \n",
      "\n",
      "Sun Protection That Won't Wear Off. Our high-performance fabric provides SPF 50+ sun protection, blocking 98% of the sun's harmful rays.' metadata={'source': '../data/OutdoorClothingCatalog_1000.csv', 'row': 618}\n"
     ]
    }
   ],
   "source": [
    "# 将刚才创建的文本向量表征存储在向量存储中\n",
    "# 使用DocArrayInMemorySearch类的from_documents方法来实现\n",
    "# 该方法接受文档列表及向量表征模型作为输入\n",
    "db = DocArrayInMemorySearch.from_documents(docs, embeddings)\n",
    "\n",
    "query = \"请推荐一件具有防晒功能的衬衫\"\n",
    "# 使用上面的向量存储来查找与传入查询类似的文本，得到一个相似文档列表\n",
    "docs = db.similarity_search(query)\n",
    "\n",
    "print(\"\\n\\033[32m返回文档的个数：\\033[0m \\n\",len(docs))\n",
    "print(\"\\n\\033[32m返回的第一个文档：\\033[0m \\n\",docs[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用查询结果构造提示来回答问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "以下是所有具有防晒功能的衬衫的总结，以 Markdown 表格形式呈现：\n",
       "\n",
       "| 编号 | 商品名称 | 防晒等级 | 材质 | 适用人群 | 主要特点总结 |\n",
       "|------|----------|-----------|--------|------------|----------------|\n",
       "| 618 | Men's Tropical Plaid Short-Sleeve Shirt | UPF 50+（SPF 50+） | 100% polyester | 男性 | 轻便透气，适合炎热天气；传统宽松剪裁，前后面罩通风设计，双前袋；抗皱、进口；提供最高级别紫外线防护。 |\n",
       "| 709 | Sunrise Tee | UPF 50+（SPF 50+） | 71% nylon, 29% polyester（外层），100% polyester（内衬） | 女性 | 略微合身剪裁，贴合身形；快速排汗、速干、抗皱；前后通风设计，低轮廓口袋，工具挂环与眼镜挂绳；内置SunSmart™防晒技术，适合钓鱼或旅行。 |\n",
       "| 293 | Girls' Ocean Breeze Long-Sleeve Stripe Shirt | UPF 50+（SPF 50+） | 尼龙 Lycra®-氨纶混纺 | 女孩/儿童 | 全覆盖长袖防晒衣，适合泳池或海滩；防褪色、快干、耐海水腐蚀；经皮肤癌基金会推荐，可搭配泳装；保持版型稳定。 |\n",
       "| 252 | Multi-Sport Performance Tee | UPF 50+（SPF 50+） | 55% polyester, 39%回收聚酯, 6%氨纶 | 通用（男女皆宜） | 活动T恤，适合运动与日常；吸湿排汗、快干、抗摩擦；抗菌防臭处理，BlueSign认证环保；肩部和背部有反光细节提升可见度；网眼侧边与后背Y形结构增强透气性；下摆略长增加覆盖。 |\n",
       "\n",
       "> ✅ **共性亮点**：\n",
       "> - 所有产品均具备 **UPF 50+ / SPF 50+** 的顶级防晒性能，能阻挡 **98% 的有害紫外线**。\n",
       "> - 防晒功能 **不随洗涤而失效**，属于高分子科技面料。\n",
       "> - 多数为 **轻质、速干、抗皱** 设计，适合户外高温环境。\n",
       "> - 均为 **进口产品**，注重功能性与舒适性结合。\n",
       "\n",
       "---  \n",
       "如需导出为文档或用于商品展示，此表格可直接复制使用。"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 合并获得的相似文本内容\n",
    "qdocs = \"\".join(docs[i].page_content for i in range(len(docs)))\n",
    "# 将合并的相似文档内容后加上问题输入到llm中\n",
    "# 这里问题是：以markdown表格的方式列出所有具有防晒功能的衬衫并总结\n",
    "response = llm.call_as_llm(f\"{qdocs} \\n 问题：请用markdown表格的方式列出所有具有防晒功能的衬衫，对每件衬衫描述进行总结\")\n",
    "display(Markdown(response))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用检索问答链来回答问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_20001/2872639494.py:13: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 1.0. Use :meth:`~invoke` instead.\n",
      "  response = qa_stuff.run(query)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "以下是具有防晒功能的衬衫列表，以 Markdown 表格形式呈现，并对每件衬衫的描述进行了总结：\n",
       "\n",
       "| 编号 | 名称 | 防晒等级 | 材料 | 主要特点总结 |\n",
       "|------|------|----------|------|--------------|\n",
       "| 709 | Sunrise Tee | UPF 50+（SPF 50+） | 71%尼龙 + 29%聚酯纤维（外层），100%聚酯（内衬） | 轻量高性能面料，快速排汗、速干、抗皱；专为炎热天气设计，适合钓鱼或旅行；内置UPF 50+防晒，高防护且持久不褪色。 |\n",
       "| 28 | Performance Plus Woven Shirt | UPF 40+（SPF 50+） | 100%尼龙 | 快速干燥（<14分钟），耐磨耐用，仿棉质感；适合远足或城市出行；具备优异的吸湿排汗与紫外线防护性能。 |\n",
       "| 396 | Perform-A-Tex Woven Shirt | UPF 40+（SPF 50+） | 100%尼龙 | 与Performance Plus同款，快干、耐磨、吸湿排汗；适合户外和城市多场景使用；提供高达98%的紫外线阻挡率。 |\n",
       "| 293 | Girls' Ocean Breeze Long-Sleeve Stripe Shirt | UPF 50+（SPF 50+） | 尼龙 Lycra®-氨纶混纺 | 女童长袖防晒衣，全幅覆盖，适合泳池或海滩；防褪色、耐海水腐蚀、保持形状；获皮肤癌基金会推荐，高效防晒。 |\n",
       "\n",
       "> ✅ **注**：所有防晒衬衫均标明“Sun Protection That Won't Wear Off”，即防晒功能不会因洗涤或使用而失效。其中，**UPF 50+** 为最高防晒等级，代表可阻挡98%以上的紫外线。"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 基于向量存储，创建检索器\n",
    "retriever = db.as_retriever()\n",
    "\n",
    "qa_stuff = RetrievalQA.from_chain_type(\n",
    "    llm = llm,\n",
    "    chain_type = \"stuff\",\n",
    "    retriever = retriever,\n",
    "    verbose = True\n",
    ")\n",
    "\n",
    "# 创建一各查询，并在此查询上运行链\n",
    "query = \"请用markdown表格的方式列出所有具有防晒功能的衬衫，对每件衬衫描述进行总结\"\n",
    "response = qa_stuff.run(query)\n",
    "display(Markdown(response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
